{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 导入工具包"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import time\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 设定数据列名"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "CONTINUOUS_COLUMNS =  [\"I\"+str(i) for i in range(1,14)] # 1-13 inclusive\n",
    "CATEGORICAL_COLUMNS = [\"C\"+str(i) for i in range(1,27)] # 1-26 inclusive\n",
    "LABEL_COLUMN = [\"clicked\"]\n",
    "\n",
    "TRAIN_DATA_COLUMNS = LABEL_COLUMN + CONTINUOUS_COLUMNS + CATEGORICAL_COLUMNS"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 定义处理格式处理函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input function configured\n"
     ]
    }
   ],
   "source": [
    "BATCH_SIZE = 400\n",
    "\n",
    "def generate_input_fn(filename, batch_size=BATCH_SIZE):\n",
    "    def _input_fn():\n",
    "        filename_queue = tf.train.string_input_producer([filename])\n",
    "        reader = tf.TextLineReader()\n",
    "        # Reads out batch_size number of lines\n",
    "        key, value = reader.read_up_to(filename_queue, num_records=batch_size)\n",
    "        \n",
    "        # 1 int label, 13 ints, 26 strings\n",
    "        cont_defaults = [ [0] for i in range(1,14) ]\n",
    "        cate_defaults = [ [\" \"] for i in range(1,27) ]\n",
    "        label_defaults = [ [0] ]\n",
    "        column_headers = TRAIN_DATA_COLUMNS\n",
    "        # The label is the first column of the data.\n",
    "        # 如果有缺失值就用默认值代替\n",
    "        record_defaults = label_defaults + cont_defaults + cate_defaults\n",
    "\n",
    "        # Decode CSV data that was just read out. \n",
    "        # Note that this does NOT return a dict, \n",
    "        # so we will need to zip it up with our headers\n",
    "        columns = tf.decode_csv(\n",
    "            value, record_defaults=record_defaults)\n",
    "        \n",
    "        # all_columns is a dictionary that maps from column names to tensors of the data.\n",
    "        all_columns = dict(zip(column_headers, columns))\n",
    "        \n",
    "        # Pop and save our labels \n",
    "        # dict.pop() returns the popped array of values; exactly what we need!\n",
    "        labels = all_columns.pop(LABEL_COLUMN[0])\n",
    "        \n",
    "        # the remaining columns are our features\n",
    "        features = all_columns \n",
    "\n",
    "        # Sparse categorical features must be represented with an additional dimension. \n",
    "        # There is no additional work needed for the Continuous columns; they are the unaltered columns.\n",
    "        # See docs for tf.SparseTensor for more info\n",
    "        for feature_name in CATEGORICAL_COLUMNS:\n",
    "            features[feature_name] = tf.expand_dims(features[feature_name], -1)\n",
    "\n",
    "        return features, labels\n",
    "\n",
    "    return _input_fn\n",
    "\n",
    "print('input function configured')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 建立输入到DNN_Linear模型的特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wide/Sparse columns configured\n"
     ]
    }
   ],
   "source": [
    "# Sparse base columns：对于类别特征，用hash编码将其转化为稀疏列\n",
    "wide_columns = []# 用于Line模型的特征\n",
    "for name in CATEGORICAL_COLUMNS:\n",
    "    wide_columns.append(tf.contrib.layers.sparse_column_with_hash_bucket(\n",
    "            name, hash_bucket_size=1000))\n",
    "\n",
    "print('Wide/Sparse columns configured')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "deep/continuous columns configured\n"
     ]
    }
   ],
   "source": [
    "# Continuous base columns：对于连续性特征，直接使用真实值。\n",
    "deep_columns = []# 用于DNN模型的特征\n",
    "for name in CONTINUOUS_COLUMNS:\n",
    "    deep_columns.append(tf.contrib.layers.real_valued_column(name))\n",
    "\n",
    "print('deep/continuous columns configured')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "wide and deep columns configured\n"
     ]
    }
   ],
   "source": [
    "# Embeddings for wide columns into deep columns：再将类别特征进行词嵌入模型，送入DNN模型。\n",
    "for col in wide_columns:\n",
    "    deep_columns.append(tf.contrib.layers.embedding_column(col, \n",
    "                                                           dimension=8))\n",
    "\n",
    "print('wide and deep columns configured')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 建立模型\n",
    "* **Wide & Deep**: Combined Linear and Deep Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model directory = models/model_WIDE_AND_DEEP_1560670723\n",
      "INFO:tensorflow:Using config: {'_task_type': None, '_task_id': 0, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x00000146901C6240>, '_master': '', '_num_ps_replicas': 0, '_num_worker_replicas': 0, '_environment': 'local', '_is_chief': True, '_evaluation_master': '', '_train_distribute': None, '_device_fn': None, '_tf_config': gpu_options {\n",
      "  per_process_gpu_memory_fraction: 1\n",
      "}\n",
      ", '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_secs': None, '_log_step_count_steps': 100, '_session_config': None, '_save_checkpoints_steps': 100, '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_model_dir': 'models/model_WIDE_AND_DEEP_1560670723'}\n",
      "estimator built\n"
     ]
    }
   ],
   "source": [
    "def create_model_dir(model_type):\n",
    "    # Returns something like models/model_WIDE_AND_DEEP_1493043407\n",
    "    return 'models/model_' + model_type + '_' + str(int(time.time()))\n",
    "\n",
    "# Specify the desired model_dir \n",
    "def get_model(model_type, model_dir):\n",
    "    print(\"Model directory = %s\" % model_dir)\n",
    "    \n",
    "    # There are more options here than shown here. \n",
    "    # We are using this to show additional checkpointing for illustrative purposes.\n",
    "    # In a real system with far more samples, you would \n",
    "    #     likely choose to save checkpoints less frequently.\n",
    "    runconfig = tf.contrib.learn.RunConfig(\n",
    "        save_checkpoints_secs=None,\n",
    "        save_checkpoints_steps = 100,\n",
    "    )\n",
    "    \n",
    "    m = None\n",
    "    \n",
    "    # Linear Classifier\n",
    "    if model_type == 'WIDE':\n",
    "        m = tf.contrib.learn.LinearClassifier(\n",
    "            model_dir=model_dir, \n",
    "            feature_columns=wide_columns)\n",
    "\n",
    "    # Deep Neural Net Classifier\n",
    "    if model_type == 'DEEP':\n",
    "        m = tf.contrib.learn.DNNClassifier(\n",
    "            model_dir=model_dir,\n",
    "            feature_columns=deep_columns,\n",
    "            hidden_units=[100, 50, 25])\n",
    "\n",
    "    # Combined Linear and Deep Classifier\n",
    "    if model_type == 'WIDE_AND_DEEP':\n",
    "        m = tf.contrib.learn.DNNLinearCombinedClassifier(\n",
    "            model_dir=model_dir,\n",
    "            linear_feature_columns=wide_columns,\n",
    "            dnn_feature_columns=deep_columns,\n",
    "            dnn_hidden_units=[100,700,50,25],\n",
    "            config=runconfig)\n",
    "        \n",
    "    print('estimator built')\n",
    "    \n",
    "    return m\n",
    "    \n",
    "\n",
    "MODEL_TYPE = 'WIDE_AND_DEEP'\n",
    "model_dir = create_model_dir(model_type=MODEL_TYPE)\n",
    "m = get_model(model_type=MODEL_TYPE, model_dir=model_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 训练模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 加载数据\n",
    "train_file = \"data/train.csv\"\n",
    "eval_file  = \"data/eval.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Casting <dtype: 'int32'> labels to bool.\n",
      "WARNING:tensorflow:Casting <dtype: 'int32'> labels to bool.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 0 into models/model_WIDE_AND_DEEP_1560670723\\model.ckpt.\n",
      "INFO:tensorflow:loss = 4.3260183, step = 2\n",
      "INFO:tensorflow:Saving checkpoints for 102 into models/model_WIDE_AND_DEEP_1560670723\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 7.59014\n",
      "INFO:tensorflow:loss = 0.45413193, step = 202 (18.034 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 204 into models/model_WIDE_AND_DEEP_1560670723\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 9.93795\n",
      "INFO:tensorflow:Saving checkpoints for 306 into models/model_WIDE_AND_DEEP_1560670723\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 15.9216\n",
      "INFO:tensorflow:loss = 0.53203994, step = 402 (13.141 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 408 into models/model_WIDE_AND_DEEP_1560670723\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 15.719\n",
      "INFO:tensorflow:Saving checkpoints for 510 into models/model_WIDE_AND_DEEP_1560670723\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 15.7461\n",
      "INFO:tensorflow:loss = 0.4790123, step = 602 (12.927 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 612 into models/model_WIDE_AND_DEEP_1560670723\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 16.7406\n",
      "INFO:tensorflow:Saving checkpoints for 714 into models/model_WIDE_AND_DEEP_1560670723\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 16.2544\n",
      "INFO:tensorflow:loss = 0.51527566, step = 802 (12.344 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 816 into models/model_WIDE_AND_DEEP_1560670723\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 16.4556\n",
      "INFO:tensorflow:Saving checkpoints for 918 into models/model_WIDE_AND_DEEP_1560670723\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 16.226\n",
      "INFO:tensorflow:loss = 0.5627922, step = 1002 (12.420 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 1020 into models/model_WIDE_AND_DEEP_1560670723\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 16.8186\n",
      "INFO:tensorflow:Saving checkpoints for 1122 into models/model_WIDE_AND_DEEP_1560670723\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 16.0741\n",
      "INFO:tensorflow:loss = 0.46116143, step = 1202 (12.390 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 1224 into models/model_WIDE_AND_DEEP_1560670723\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 16.1289\n",
      "INFO:tensorflow:Saving checkpoints for 1326 into models/model_WIDE_AND_DEEP_1560670723\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 15.8981\n",
      "INFO:tensorflow:loss = 0.5011395, step = 1402 (12.736 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 1428 into models/model_WIDE_AND_DEEP_1560670723\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 16.4845\n",
      "INFO:tensorflow:Saving checkpoints for 1530 into models/model_WIDE_AND_DEEP_1560670723\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 15.0511\n",
      "INFO:tensorflow:loss = 0.56771296, step = 1602 (13.029 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 1632 into models/model_WIDE_AND_DEEP_1560670723\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 14.4208\n",
      "INFO:tensorflow:Saving checkpoints for 1734 into models/model_WIDE_AND_DEEP_1560670723\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 15.5665\n",
      "INFO:tensorflow:loss = 0.4926152, step = 1802 (13.518 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 1836 into models/model_WIDE_AND_DEEP_1560670723\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 14.7361\n",
      "INFO:tensorflow:Saving checkpoints for 1938 into models/model_WIDE_AND_DEEP_1560670723\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 15.7146\n",
      "INFO:tensorflow:loss = 0.49656898, step = 2002 (13.394 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 2002 into models/model_WIDE_AND_DEEP_1560670723\\model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.49656898.\n",
      "fit done\n",
      "Wall time: 3min 1s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# This can be found with\n",
    "# wc -l train.csv\n",
    "train_sample_size = 800000\n",
    "train_steps = train_sample_size/BATCH_SIZE # 800000/400 = 2000\n",
    "\n",
    "m.fit(input_fn=generate_input_fn(train_file, BATCH_SIZE), steps=train_steps)\n",
    "\n",
    "print('fit done')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 验证模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Casting <dtype: 'int32'> labels to bool.\n",
      "WARNING:tensorflow:Casting <dtype: 'int32'> labels to bool.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "INFO:tensorflow:Starting evaluation at 2019-06-16-07:45:13\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from models/model_WIDE_AND_DEEP_1560670723\\model.ckpt-2002\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Evaluation [50/500]\n",
      "INFO:tensorflow:Evaluation [100/500]\n",
      "INFO:tensorflow:Evaluation [150/500]\n",
      "INFO:tensorflow:Evaluation [200/500]\n",
      "INFO:tensorflow:Evaluation [250/500]\n",
      "INFO:tensorflow:Evaluation [300/500]\n",
      "INFO:tensorflow:Evaluation [350/500]\n",
      "INFO:tensorflow:Evaluation [400/500]\n",
      "INFO:tensorflow:Evaluation [450/500]\n",
      "INFO:tensorflow:Evaluation [500/500]\n",
      "INFO:tensorflow:Finished evaluation at 2019-06-16-07:45:27\n",
      "INFO:tensorflow:Saving dict for global step 2002: accuracy = 0.7657, accuracy/baseline_label_mean = 0.251165, accuracy/threshold_0.500000_mean = 0.7657, auc = 0.72143906, auc_precision_recall = 0.47333822, global_step = 2002, labels/actual_label_mean = 0.251165, labels/prediction_mean = 0.25214505, loss = 0.502732, precision/positive_threshold_0.500000_mean = 0.6372589, recall/positive_threshold_0.500000_mean = 0.15587363\n",
      "evaluate done\n",
      "Accuracy: 0.7657\n",
      "{'loss': 0.502732, 'accuracy': 0.7657, 'labels/prediction_mean': 0.25214505, 'labels/actual_label_mean': 0.251165, 'accuracy/baseline_label_mean': 0.251165, 'auc': 0.72143906, 'auc_precision_recall': 0.47333822, 'accuracy/threshold_0.500000_mean': 0.7657, 'precision/positive_threshold_0.500000_mean': 0.6372589, 'recall/positive_threshold_0.500000_mean': 0.15587363, 'global_step': 2002}\n",
      "Wall time: 24.9 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "eval_sample_size = 200000 \n",
    "eval_steps = eval_sample_size/BATCH_SIZE \n",
    "\n",
    "results = m.evaluate(input_fn=generate_input_fn(eval_file), \n",
    "                     steps=eval_steps)\n",
    "print('evaluate done')\n",
    "\n",
    "print('Accuracy: %s' % results['accuracy'])\n",
    "print(results)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "DNN_Linear模型： dnn_hidden_units=[100, 70, 50, 25]得到的结果   \n",
    "Accuracy:0.7657    \n",
    "loss: 0.502732  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
